package io.renren.modules.job.task.crawl;

import cn.hutool.core.date.DateUtil;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.XpathSelector;

import java.util.Date;
import java.util.List;

/**
 * @auther: Freeman Liu
 * @date: 2020/11/4 15:01
 */
@Component
public class NewsProcessor implements PageProcessor {

    // 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
    private Site site = Site.me().setRetryTimes(2).setSleepTime(1000);

    @Override
    public void process(Page page) {

        Html pageHtml = page.getHtml();
        String title = pageHtml.xpath("/html/body/div[7]/h1/text()").toString();
        String newsTime = pageHtml.xpath("/html/body/div[7]/div[1]/span/text()").toString();
//        String content = pageHtml.xpath("/html/body/div[7]/article/div[2]/p/text()").get();
        List<String> contentList = pageHtml.selectDocumentForList(new XpathSelector("/html/body/div[7]/article/div[2]/p/text()"));
        StringBuilder content = new StringBuilder();
        for (String s : contentList) {
            content.append(s).append("\n");
        }

//        System.out.println(title);
//        System.out.println(content);
//        System.out.println(newsTime);

        if (StringUtils.isAnyBlank(title, newsTime, content)) {
            page.setSkip(true);
        } else {
            if (isSameDay(newsTime)) {
                page.putField("title", title);
                page.putField("content", content.toString());
                page.putField("newsTime", DateUtil.parse(newsTime));
            } else {
                page.setSkip(true);
            }
        }

        //        page.addTargetRequests(page.getHtml().links().all());  //爬取页面中所有的链接
        page.addTargetRequests(
                page.getHtml()
                        .links()
                        .regex("http://www.cnncw.cn/[0-9]+/[0-9]+/[0-9]+.shtml")
                        .all()); //爬取详情页面
    }

    @Override
    public Site getSite() {
        return site;
    }

//    public static void main(String[] args) {
//        Spider.create(new NewsProcessor())
//                .addUrl("https://www.csdn.net/nav/python")
////                .addPipeline(new ConsolePipeline())
//                .addPipeline(new NewsPipeline())
////                .runAsync();
//                .run();
//    }

    private boolean isSameDay(String dateStr) {
        String newsDate = dateStr.split(" ")[0];
        String today = DateUtil.format(new Date(), "yyyy-MM-dd");
        return newsDate.equals(today);
    }

}
